package me.zhangjin.lucene.demo.analyzer;

import java.io.IOException;
import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;


public class StdAnalyzer {
    private static String strCh = "中华人民共和国简称中国, 是一个有13亿人口的国家";
    private static String strEn = "Dogs can not achieve a place,eyes can reach; ";

    public static void main(String[] args) throws IOException {
        System.out.println("StandardAnalyzer对中文分词:");
        stdAnalyzer(strCh);
        System.out.println("StandardAnalyzer对英文分词:");
        stdAnalyzer(strEn);

    }

    public static void stdAnalyzer(String str) throws IOException {
        // 标准分词器 StandardAnalyzer
        Analyzer analyzer = new StandardAnalyzer();
        // 输入器
        StringReader reader = new StringReader(str);
        // 分词完成得到一个分词流
        TokenStream toStream = analyzer.tokenStream(str, reader);
        // 重置流的位置
        toStream.reset();

        // 得到 Char Term 信息
        CharTermAttribute teAttribute = toStream.getAttribute(CharTermAttribute.class);

        System.out.println("分词结果:");

        // 每进一位，输出一个 Char Term
        while (toStream.incrementToken()) {
            System.out.print(teAttribute.toString() + "|");
        }
        System.out.println("\n");
        analyzer.close();
    }
}
